# -*- coding: utf-8 -*-
# @Time    : 2018/10/18 10:04
# 插入mongo
import os
import sys
path = os.path.abspath(os.path.dirname(os.getcwd()))
sys.path.append(path)
import pymongo
import threading
import time,random,requests
import config
import log
from datetime import datetime
from mongoFild import insert_mongo
from time import sleep
from multiprocessing import Process, Queue, Pool
new_url_biying = config.new_li_db_biying
new_path_biying = config.new_path_biying
headers = {
            'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/69.0.3497.12 Safari/537.36',
}

class FindSource(object):
    def __init__(self) -> None:
        # self.insert_mongo = insert_mongo()
        self.B = 0
        self.pro = self.get_pro()
        super().__init__()

    def get_pro(self):
        db_one = config.con_proxy
        li_db_one = db_one['proxies_ip']
        xx_pro = []
        li_db_two = li_db_one.find()
        for i in li_db_two:
            ip = i['ip']
            xx_pro.append({"http":ip})
        return xx_pro

    def is_chinese(self,uchar):
        if uchar >= u'\u4E00' and uchar <= u'\u9fff':
            return True
        else:
            return False

    def is_english(self,uchar):
        if ( uchar >= u'\u0041' and uchar <= u'\u005A' ) or ( uchar >= u'\u0061' and uchar <= u'\u007A'):
            return True
        else:
            return False

    def find_str(self, zifuchuan):
        try:
            new_zifu = zifuchuan[:int(len(zifuchuan)*0.4)]
            china_num = 0
            english_num = 0
            th_num = 0
            for i in new_zifu:
                # 除字符
                if i.isalpha():
                    if self.is_english(i):
                        english_num += 1
                    elif self.is_chinese(i):
                        china_num += 1
                    else:
                        th_num =1
                        break
            if th_num:
                return False
            elif english_num>china_num:
                return True
            elif china_num>=english_num:
                return True
            else:
                return False
        except:
            return False

    def delete_mongo(self, proxies):
        print("stare : %s  :%s  :%s"%(os.getpid(), threading.current_thread().name, proxies))
        new_url_biying = config.new_li_db_biying
        lll = 0
        while True:
            try:
                # 关键词
                item = new_url_biying.find_and_modify({'state': 0}, {'$set': {"state": 1}})
                if not item:
                    return
                item['state'] = 1
                lll += 1
                if lll %1000 == 0:
                    print(lll)
                    print("修改state")

                try:
                    summary = item['summary']
                    print(summary)
                    #验证summary的语言
                    source = self.find_str(summary)
                except:
                    continue
                # 下载文档
                if source:
                    self.find_one(item, proxies)
            except Exception as e:
                new_url_biying = config.new_li_db_biying
                print("%s, %s"%(e,sys._getframe().f_lineno))
                print("已经重新新连接")

    def find_one(self, item, proxies):
        #获取图片的连接
        url_link = item['url']
        # 下载图片的连接
        if str(url_link).find("http")>-1:
            try:
                try:
                    req = requests.get(url_link, headers=headers, proxies = proxies, timeout=7)
                except:
                    return
                if req.status_code == 200:
                    url_md5 = item['url_md5']
                    path = self.find_file_name(url_md5)
                    if path:
                        houzhui = self.find_houzhu(url_link,item['type'])
                        new_path = str(path) + "\\" + url_md5 + ".%s"%(houzhui)
                        fo = open(new_path, 'wb')
                        fo.write(req.content)
                        fo.close()
                        source = self.insert_mongo_yuanshuju(item, new_path)
                        if source:
                            new_url_biying.update({'url_md5':item['url_md5']}, {'$set': {"state": 2}})
                            self.B += 1
                            if self.B % 10 == 0:
                                print("%s  ：修改"%url_md5)
            except Exception as e:
                print("%s, %s"%(e,sys._getframe().f_lineno))

    def insert_mongo_yuanshuju(self, item, path):
        try:
            new_path = str(path).split("\\",1)[-1]
            item["path"] = new_path
            del item["state"]
            try:
                del item["title"]
            except:
                pass
            try:
                del item['summary']
            except:
                pass
            del item["type"]
            del item["q"]
            new_path_biying.insert(item)
            return  True
        except Exception as e:
            print("插入失败 %s"%e)
            return False

    def find_houzhu(self, url_link, type):
        houzhu = str(url_link).split(".")[-1]
        if len(houzhu)>4:
            return type
        else:
            if houzhu.find("com")>-1 or houzhu.find("/")>-1:
                return type
            else:
                return houzhu
    def find_file_name(self,name_md5):
        try:
            file_zimu = name_md5[:3]
            path = str((config.file_biying_file)%(datetime.now().strftime('%Y%m%d')) + str("%s\\%s")%(file_zimu[0],file_zimu[1:]))
            file_name = self.open(path)
            if not file_name:
                sys.exit()
            else:
                return path
        except Exception as e:
            print("%s, %s"%(e,sys._getframe().f_lineno))

    def open(self, path):
        #新建文件夹
        isExists = os.path.exists(path)
        # 调用函数
        if not isExists:
            try:
                os.makedirs(path)
                print("%s  open file_name:%s" % (os.getpid(), path))
                return True
            except Exception as e:
                print("%s  open file_name:%s" % (e, path))
                return self.open(path)
        else:
            return True

    def proce(self):
        trader = []
        for i in range(10):
            proxies = self.pro[i]
            pr = Process(target=self.delete_carw, args=(proxies,))
            sleep(0.5)
            pr.start()
            trader.append(pr)
        for i in trader:
            i.join()
        print('proce this is pid: %s' % os.getpid())

    def delete_carw(self, proxies):
        threads = []
        for i in range(2):
            t = threading.Thread(target=self.delete_mongo, name='LoopThread_%s'% i, args=(proxies,))
            threads.append(t)
        for t in threads:
            sleep(0.5)
            t.start()
        for t in threads:
            # 多线程多join的情况下，依次执行各线程的join方法, 这样可以确保主线程最后退出， 且各个线程间没有阻塞
            t.join()
        print('delete this is pid: %s' % os.getpid())
        return


if __name__ == '__main__':
    # lin_sql.creat_fulltext()
    FindSource().proce()





